#ifndef __THREADING_HPP__
#define __THREADING_HPP__

#include <iostream>
#include <sstream>
#include <string>
#include <cassert>
#include <mutex>
#include <thread>

#include <pthread.h>
#include <unistd.h>
#include <numa.h>
#include <numaif.h>
#include <linux/sched.h>

class Threading {
  static inline std::mutex &display_mutex() {
    static std::mutex the_mutex;
    return the_mutex;
  }
  static inline std::mutex &count_mutex() {
    static std::mutex the_mutex;
    return the_mutex;
  }
  static inline std::mutex &thread_count_mutex() {
    static std::mutex the_mutex;
    return the_mutex;
  }
  static inline void dump(std::ostream & os, std::string const &s) {
    display_mutex().lock();
    os << s;
    display_mutex().unlock();
  }
 
public:
  static void dumpThreadInfo(std::ostream & os, std::string const &s);
  static void dumpMemInfo(std::ostream & os, void *addr);

  static int threadCountInc(bool pinned) {
    static int count = -1;

    assert(!pinned);
    thread_count_mutex().lock();
    count++;
    thread_count_mutex().unlock();
  
    return count;
  }

  // Just increments a counter and pins thread to the next processor modulo NUM_PROCS
  // Later we can extend this behavior
  static void pinThread(int cpu) {
#define NUM_PROCS 12
    cpu_set_t cpuSet;
    
    CPU_ZERO(&cpuSet);
    CPU_SET(cpu, &cpuSet);
    std::cerr << "PIN thread  0x" << std::hex << std::this_thread::get_id()
              << " to cpu " << std::dec << cpu << "\n";
    int res = pthread_setaffinity_np(pthread_self(), sizeof(cpuSet), &cpuSet);
    if (res != 0) {
      std::cerr << "sched_setaffinity failed, exit with code " << errno << std::endl;
      exit(errno);
    }
  }

  // This is very problem and processor specific.
  // Atm, just use a proof of concept implementation, slab0 goes to node 0 and
  // slab1 goes to node 1.
  template<typename ARRAY_TYPE>
  static void pinPages(void *A, int SLABS, int ARRAY_SIZE, ARRAY_TYPE t) {    
    //
    // Can't make membind work properly atm
    //
    // nodemask_t mask;
    // nodemask_zero(&mask);
    // nodemask_set_compat(&mask, 1);
    // int mbind(void *addr, unsigned long len, int mode,
    //           unsigned long *nodemask, unsigned long maxnode,
    //           unsigned flags);
    // 
    // uint64_t ptrInt = (uint64_t)&A[0][0][0][0][0];
    // uint64_t m = 0xfffffffffffff800;
    // ptrInt = ptrInt & m;
    // void *ptr = (void *)ptrInt;
    // cerr << "mbind 1 @0x" << std::hex << ptr << std::endl;
    // int res = mbind(ptr, Npadded * Npadded * Npadded * sizeof(double), 
    //                 MPOL_DEFAULT // | MPOL_BIND
    //                 , mask.n, 1, 
    //                 MPOL_MF_MOVE // | MPOL_MF_STRICT
    //                 );
    // if (res != 0) {
    //   cerr << "mbind 1 failed, exit with code " << errno << std::endl;
    //   exit(errno);
    // }

    // Use move_pages instead
    int pageSize = Threading::getPageSize();
    int ind, count=0;
    for (int arr=0; arr<SLABS; arr++) {
      for (ind=0; ind < ARRAY_SIZE * sizeof(decltype(t)); ind+=pageSize) {
        uint64_t ptrInt = (uint64_t)(((char *)A) + 
                                     arr*ARRAY_SIZE*sizeof(decltype(t)) + ind);
        uint64_t m = ~(pageSize-1);
        ptrInt = ptrInt & m;
        void *ptr = (void *)ptrInt;

        // Touch one element of the page so that it is pinned before we can move it.
        char *c = (char *)ptrInt;
        *c = 0.0;

        int status;
        int node = arr;
        long ret = move_pages(0, 1, &ptr, &node, &status, MPOL_MF_MOVE);
#if (VERBOSE >= 10)
        std::cerr << "status for page num=" << count++ << " with ind=" << std::hex 
             << " " << ptr << " is " << std::dec << status << " and ret is " << ret 
             << "\n" ;
#endif
        if (status < 0) exit(0);
      }
    }
  }

  static void assertPageSize(int size) {
    if (size != getpagesize()) {
      std::cerr << "Page Size = " << getpagesize() << " and NOT " << size << "\n";
      assert(false);
    }
  }

  static int getPageSize() {
    return getpagesize();
  }
};
#endif // __THREADING_HPP__
